from itertools import *
from operator import itemgetter

print "hello world"

popularQueryTermPair1MDict = {}
queryTermPairFreqList = []
histogramForQueryTermPairFreqDict = {}
histogramForQueryTermPairFreqList = []

# key: query term 
# value: a list for the second parts
queryTermWithTheirSecondPartsDict = {}
queryTermWithTheirSecondPartsList = []


# for production
inputQueryFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries"
# for debug
# inputQueryFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries_customize_2_queries_for_debug"
inputQueryHandler = open(inputQueryFileName,"r")

# for production
outputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairFreqFor100KQueries_NEW20130613"
# for debug
# outputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairFreqFor100KQueries_customize_2_queries_for_debug"
outputFileHandler1 = open(outputFileName1,"w")

# for production
outputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairHistogramFor100KQueries_NEW20130613"
# for debug
# outputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairHistogramFor100KQueries_customize_2_queries_for_debug"
outputFileHandler2 = open(outputFileName2,"w")

# for production
outputFileName3 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermWithTheirSecondPartsFor100KQueries_NEW20130613"
# for debug
# outputFileName3 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermWithTheirSecondPartsFor100KQueries_customize_2_queries_for_debug"
outputFileHandler3 = open(outputFileName3,"w")

for line in inputQueryHandler.readlines():
    elements = line.strip().split(":")
    queryID = int(elements[0])
    
    data = elements[1]
    data = data.lower()
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    
    queryContent = data
    
    queryContentElements = queryContent.strip().split(" ")
    queryContentElementsWithoutEmptyString = []
    for term in queryContentElements:
        if term.strip() != "":
            queryContentElementsWithoutEmptyString.append(term)
    queryContentElementsWithoutEmptyString.sort(cmp=None, key=None, reverse=False)
     
    # for debug ONLY 
    # print queryContentElementsWithoutEmptyString
    # print len(queryContentElementsWithoutEmptyString)
    
    '''
    print "just for checking begins..."
    print queryContentElementsWithoutEmptyString
    for currentTuple in permutations(queryContentElementsWithoutEmptyString,2):
        print currentTuple
    print "just for checking ends."
    '''
    
    # add the logic to support ONE term query.
    # Updated by Wei 2013/06/13
    if len(queryContentElementsWithoutEmptyString) == 1:
        queryTermWithTheirSecondPartsDict[ queryContentElementsWithoutEmptyString[0] ] = []
    
    
    
    for currentTuple in permutations(queryContentElementsWithoutEmptyString,2):
        (term1,term2) = currentTuple
        # big mistake here
        key = term1 + "_" + term2
        
        # updated the data structure popularQueryTermPair1MDict
        if key not in popularQueryTermPair1MDict:
            popularQueryTermPair1MDict[key] = 1
        else:
            popularQueryTermPair1MDict[key] += 1

        # updated the data structure queryTermWithTheirSecondPartsDict
        if term1 not in queryTermWithTheirSecondPartsDict:
            queryTermWithTheirSecondPartsDict[term1] = []
            queryTermWithTheirSecondPartsDict[term1].append( term2 )
        else:
            if term2 not in queryTermWithTheirSecondPartsDict[term1]:
                queryTermWithTheirSecondPartsDict[term1].append(term2)
            else:
                pass

print "----->","len(popularQueryTermPair1MDict):",len(popularQueryTermPair1MDict)

'''
# checking point
print "popularQueryTermPair1MDict['education_for']:",popularQueryTermPair1MDict["education_for"]
print "popularQueryTermPair1MDict['for_education']:",popularQueryTermPair1MDict["for_education"]
'''

for key in popularQueryTermPair1MDict:
    tuple = (key,popularQueryTermPair1MDict[key])
    queryTermPairFreqList.append(tuple)

print "----->","len(queryTermPairFreqList):",len(queryTermPairFreqList)

# value = 0, sorted by the lexicographic order
# value = 1, sorted by the freq
queryTermPairFreqList.sort(cmp=None, key=itemgetter(1), reverse=True)

for tuple in queryTermPairFreqList:
    (key,freq) = tuple
    outputFileHandler1.write(key + " " + str(freq) + "\n")
    if freq not in histogramForQueryTermPairFreqDict:
        histogramForQueryTermPairFreqDict[freq] = 1
    else:
        histogramForQueryTermPairFreqDict[freq] += 1

for freq in histogramForQueryTermPairFreqDict:
    histogramForQueryTermPairFreqList.append( (freq, histogramForQueryTermPairFreqDict[freq]) )

histogramForQueryTermPairFreqList.sort(cmp=None, key=itemgetter(0), reverse=False)
for tuple in histogramForQueryTermPairFreqList:
    (freq,freqOfFreq) = tuple
    outputFileHandler2.write(str(freq) + " " + str(freqOfFreq) + "\n")
    histogramForQueryTermPairFreqList
    # print tuple

for queryTerm in queryTermWithTheirSecondPartsDict:
    queryTermWithTheirSecondPartsList.append(queryTerm)

queryTermWithTheirSecondPartsList.sort(cmp=None, key=None, reverse=False)

for queryTerm in queryTermWithTheirSecondPartsList:
    newLine = queryTerm + " " + str( len( queryTermWithTheirSecondPartsDict[queryTerm] ) ) + " "
    for secondPartTerm in queryTermWithTheirSecondPartsDict[queryTerm]:
        newLine += secondPartTerm + " "
    newLine = newLine.strip() + "\n"
    outputFileHandler3.write(newLine)
    
inputQueryHandler.close()
outputFileHandler1.close()
outputFileHandler2.close()
outputFileHandler3.close()





